/*
 *  url_seed.cpp
 *  crawler
 *
 *  Created by Roy Zuo on May 13, 2010.
 *  Copyright 2010 O2 Micro, Inc. All rights reserved.
 *
 */

#include "url_seed.hpp"
#include <iostream>
#include <stdexcept>

#include <boost/regex.hpp>

UrlSeed::UrlSeed ( const string& db_file, const string protocol, const string& host, const vector< string >& paths, const vector< string >& expressions )
: _db_filename ( db_file ), _procotol ( protocol ), _host ( host ), _db_connection ( 0 )
{
	create_database ();

	for ( vector< string >::const_iterator iter = paths.begin(); iter != paths.end(); iter ++ )
	{
		string url = protocol + "://" + _host + *iter;
		set_url ( url, 0 );
	}

	for ( vector< string >::const_iterator iter = expressions.begin(); iter != expressions.end(); iter ++ )
	{
		_expressions.push_back ( *iter );
	}
}

UrlSeed::~UrlSeed ()
{
	if ( _db_connection )
	{
		try
		{
			_db_connection->close();
		}
		catch ( exception& ex )
		{
			cerr << "Exception Occured: " << ex.what() << endl;
		}
		
		delete _db_connection;
		_db_connection = 0;
	}
}

void UrlSeed::create_database ()
{
	try
	{
		_db_connection = new sqlite3_connection ( _db_filename.c_str() );
		int count = _db_connection->executeint("select count(*) from sqlite_master where name='t_urls';");
		if(count==0) 
		{
			_db_connection->executenonquery ( SQL_CREATE_URLS_TABLE );
		}
		else
		{
			_db_connection->executenonquery ( SQL_RESET_URLS );
		}
	}
	catch ( exception &ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}
	
}

void UrlSeed::search_content ( const string& url, const string& page_content, int category, const string& expr )
{
	string::const_iterator start, end;
	start = page_content.begin();
	end = page_content.end(); 

	boost::regex expression ( expr );
	boost::match_results<std::string::const_iterator> what;
	boost::match_flag_type flags = boost::match_default;
	
	while(boost::regex_search(start, end, what, expression, flags))   
	{
		string found_url = "";

		for ( unsigned int i=1; i < what.size(); i++ )
		{
			 found_url += string ( what[i].first, what[i].second );
		}

		string protocol, host, dir, file;
		split_url( url, protocol, host, dir, file );

		boost::regex expr1 ( "^(http[s]*)://(.*?)(/.*?)$" );
		boost::regex expr2 ( "^(/.*?)$" );
		boost::regex expr3 ( "^([^/]+)$" );

		boost::cmatch cwhat;
		string real_url = "";

		if ( boost::regex_match ( found_url.c_str(), cwhat, expr1 ) )
		{
			real_url = found_url;
		}
		else if ( boost::regex_match ( found_url.c_str(), cwhat, expr2 ) )
		{
			real_url = protocol + "://" + host + found_url;
		}
		else if ( boost::regex_match ( found_url.c_str(), cwhat, expr3 ) )
		{
			real_url = protocol + "://" + host + dir + found_url;
		}
		
		if ( real_url.length() > 0 )
		{
			set_url ( real_url, category );
		}
		
		// update search position:
		start = what[0].second;      
		// update flags:
		flags |= boost::match_prev_avail;
		flags |= boost::match_not_bob;
	}
}

void UrlSeed::unread_urls( vector < string >& urls )
{
	try
	{
		sqlite3_command st ( *_db_connection, SQL_SELECT_URLS_WITH_STATUS );
		st.bind ( 1, 0 );
		sqlite3_reader dr = st.executereader();

		while(dr.read())
		{
			string url = dr.getstring( 0 );
			urls.push_back ( url );
		}
	}
	catch ( exception& ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}
}

void UrlSeed::set_url( const string& url, int category )
{
	try
	{
		sqlite3_command st ( *_db_connection, SQL_SELECT_URLS_COUNT_WITH_URL );
		st.bind ( 1, url.c_str(), url.length() );
		int count = st.executeint();
		if ( count == 0 )
		{
			sqlite3_transaction trans ( *_db_connection );

			{
				sqlite3_command cmd  ( *_db_connection, SQL_INSERT_URLS );
				cmd.bind ( 1, url.c_str(), url.length() );
				cmd.bind ( 2, category );
				cmd.executenonquery();
			}

			trans.commit();
			
		}
	}
	catch ( exception& ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}
}

void UrlSeed::update ( const string& url, int status )
{
	string real_url = "";

	boost::regex expr1 ( "^(http[s]*)://(.*?)(/.*?)$" );
	boost::regex expr2 ( "^(/.*?)$" );

	boost::cmatch what;
	if ( boost::regex_match ( url.c_str(), what, expr1 ) )
	{
		real_url = url;
	}
	else if ( boost::regex_match ( url.c_str(), what, expr2 ) )
	{
		real_url = _procotol + "://" + _host + url;
	}

	try
	{
		sqlite3_transaction trans ( *_db_connection );

		{
			sqlite3_command cmd  ( *_db_connection, SQL_UPDATE_URLS );
			cmd.bind ( 1, status );
			cmd.bind ( 2, real_url.c_str(), real_url.length() );
			cmd.executenonquery();
		}

		trans.commit();

	}
	catch ( exception& ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}
}

void UrlSeed::update ( const string& url, const string& page_header, const string& page_content )
{
	string real_url = "";

	boost::regex expr1 ( "^(http[s]*)://(.*?)(/.*?)$" );
	boost::regex expr2 ( "^(/.*?)$" );
	

	string protocol, host, dir, file;

	boost::cmatch what;
	if ( boost::regex_match ( url.c_str(), what, expr1 ) )
	{
		real_url = url;
	}
	else if ( boost::regex_match ( url.c_str(), what, expr2 ) )
	{
		real_url = _procotol + "://" + _host + url;
	}

	try
	{
		sqlite3_transaction trans ( *_db_connection );

		{
			sqlite3_command cmd  ( *_db_connection, SQL_UPDATE_URLS );
			cmd.bind ( 1, (int)1 );
			cmd.bind ( 2, real_url.c_str(), real_url.length() );
			cmd.executenonquery();
		}

		trans.commit();

	}
	catch ( exception& ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}

	int category = 1;
	for ( vector< string >::const_iterator iter = _expressions.begin(); iter != _expressions.end(); iter ++ )
	{
		search_content ( real_url, page_content, category, *iter );
		category += 1;
	}
}

void UrlSeed::split_url ( const string& url, string& protocol, string& host, string& dir, string& file )
{
	boost::regex expr1 ( "^(http[s]*)://(.*?)(/.+/)(.*)$" );
	boost::regex expr2 ( "^(http[s]*)://(.*?)(/)([^/]*)$" );
	boost::regex expr3 ( "^(/.+/)(.*)$" );
	boost::regex expr4 ( "^([^/]+)$" );

	boost::cmatch what;
	if ( boost::regex_match ( url.c_str(), what, expr1 ) )
	{
		protocol = string ( what[1].first, what[1].second );
		host = string ( what[2].first, what[2].second );
		dir = string ( what[3].first, what[3].second );
		file = string ( what[4].first, what[4].second );
	}
	else if ( boost::regex_match ( url.c_str(), what, expr2 ) )
	{
		protocol = string ( what[1].first, what[1].second );
		host = string ( what[2].first, what[2].second );
		dir = string ( what[3].first, what[3].second );
		file = string ( what[4].first, what[4].second );
	}
	else if ( boost::regex_match ( url.c_str(), what, expr3 ) )
	{
		protocol = _procotol;
		host = _host;
		dir = string ( what[1].first, what[1].second );
		file = string ( what[2].first, what[2].second );
	}
	else if ( boost::regex_match ( url.c_str(), what, expr4 ) )
	{
		protocol = _procotol;
		host = _host;
		dir = _dir;
		file = string ( what[1].first, what[1].second );
	}
}

#if (defined __URL_SEED_UNITTEST__)

#include <sstream>
#include <boost/algorithm/string.hpp>
#include "webpage_hunter.hpp"

int main ( int argc, char* argv[] )
{
	vector < string > paths;
	paths.push_back( string ( "/" ) );

	vector < string > exprs;
	exprs.push_back ( string ( "href=[\"']+(http://|https://|)([a-z]*\\.sina\\.com\\.cn.*?)[\"']+" ) );

	UrlSeed seed ( "test.db", "http", "www.sina.com.cn", paths, exprs);

	boost::asio::io_service io_service;
	HttpHeaderPairs headers;

	stringstream header_stream;
	stringstream content_stream;

	WebpageHunter h ( io_service, &header_stream, &content_stream, "www.sina.com.cn", "http", "www.sina.com.cn", "/", headers );
	io_service.run();

	string page_content = content_stream.str();

	boost::to_lower ( page_content );
	boost::trim ( page_content );
	boost::erase_all ( page_content, "\r" );
	boost::erase_all ( page_content, "\n" );

	boost::regex ex;
	ex.assign ( "[\\s\\t]+" );

	std::ostringstream t(std::ios::out | std::ios::binary);
	std::ostream_iterator<char> oi(t);
	boost::regex_replace(oi, page_content.begin(), page_content.end(), ex, " ", boost::match_default | boost::format_all);
	string formatted_page_content(t.str());

	seed.update ( "http://www.sina.com.cn/", header_stream.str(), formatted_page_content );

	paths.clear();
	seed.unread_urls ( paths );

	for ( vector< string >::const_iterator iter = paths.begin(); iter != paths.end(); iter ++ )
	{
		cout << *iter << "\n";
	}

	return 0;
}

#endif
